package com.chenjun.run;

import java.util.ArrayList;
import java.util.List;

import org.apache.commons.lang3.StringUtils;

import com.chenjun.datastructure.URLPool;
import com.chenjun.parse.DownLoadFile;
import com.chenjun.parse.HtmlParse;

public class Main
{
	private static URLPool urlPool = new URLPool();
	private static HtmlParse htmlParse = new HtmlParse();
	private static DownLoadFile downLoadFile = new DownLoadFile();

	//---------------核心算法,广度优先搜索算法---------------------
	public static void runRoot(String rootUrl)
	{
		// 1 .root网页
		String responseBody;
		List<String> strs = new ArrayList<String>(0);
		// 根节点入队列
		urlPool.addUnVisitedUrl(rootUrl); 
		//如果没访问过的URL节点不为空，程序继续执行
		while( ! urlPool.unVisitedUrlIsEmpty())
		{
			//从队列中取出队列头节点
			String headUrl = urlPool.getUnVisitedUrl().getQueueHead();
			responseBody = downLoadFile.downLoadFile(headUrl, 5000);
			// 已访问的URL缓存到hashSet中
			urlPool.getVisitedUrl().add(headUrl); 
			//出列
			urlPool.getUnVisitedUrl().outQueue(headUrl); 
			if(StringUtils.isNotBlank(responseBody))
			{
				//获取html文档中所有的超链接
				strs = htmlParse.getHyperLinks(responseBody);
				//遍历超链接这个html里面的所有html连接list
				for(String s : strs)
				{
					if(s.startsWith("http://"))
					{
						//新增到未访问的URL队列中 已经访问过的url不再次访问
						urlPool.addUnVisitedUrl(s);
					}
				}
			}
		}
	}

	public static void main(String[] args)
	{
		runRoot("http://www.oschina.net");
	}

}
